{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!pip install autokeras\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Load Images from Disk\n",
    "If the data is too large to put in memory all at once, we can load it batch by batch into memory from disk with tf.data.Dataset.\n",
    "This [function](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/image_dataset_from_directory) can help you build such a tf.data.Dataset for image data.\n",
    "\n",
    "First, we download the data and extract the files.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import autokeras as ak\n",
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n",
    "local_file_path = tf.keras.utils.get_file(origin=dataset_url, \n",
    "                                          fname='image_data', \n",
    "                                          extract=True)\n",
    "# The file is extracted in the same directory as the downloaded file.\n",
    "local_dir_path = os.path.dirname(local_file_path)\n",
    "# After check mannually, we know the extracted data is in 'flower_photos'.\n",
    "data_dir = os.path.join(local_dir_path, 'flower_photos')\n",
    "print(data_dir)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "The directory should look like this. Each folder contains the images in the same class.\n",
    "\n",
    "```\n",
    "flowers_photos/\n",
    "  daisy/\n",
    "  dandelion/\n",
    "  roses/\n",
    "  sunflowers/\n",
    "  tulips/\n",
    "```\n",
    "\n",
    "We can split the data into training and testing as we load them.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "img_height = 180\n",
    "img_width = 180\n",
    "\n",
    "train_data = ak.image_dataset_from_directory(\n",
    "    data_dir,\n",
    "    # Use 20% data as testing data.\n",
    "    validation_split=0.2,\n",
    "    subset=\"training\",\n",
    "    # Set seed to ensure the same split when loading testing data.\n",
    "    seed=123,\n",
    "    image_size=(img_height, img_width),\n",
    "    batch_size=batch_size)\n",
    "\n",
    "test_data = ak.image_dataset_from_directory(\n",
    "    data_dir,\n",
    "    validation_split=0.2,\n",
    "    subset=\"validation\",\n",
    "    seed=123,\n",
    "    image_size=(img_height, img_width),\n",
    "    batch_size=batch_size)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "Then we just do one quick demo of AutoKeras to make sure the dataset works.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "clf = ak.ImageClassifier(overwrite=True, max_trials=1)\n",
    "clf.fit(train_data, epochs=1)\n",
    "print(clf.evaluate(test_data))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Load Texts from Disk\n",
    "You can also load text datasets in the same way.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "dataset_url = \"http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\"\n",
    "\n",
    "local_file_path = tf.keras.utils.get_file(\n",
    "    fname=\"text_data\", \n",
    "    origin=dataset_url, \n",
    "    extract=True,\n",
    ")\n",
    "# The file is extracted in the same directory as the downloaded file.\n",
    "local_dir_path = os.path.dirname(local_file_path)\n",
    "# After check mannually, we know the extracted data is in 'aclImdb'.\n",
    "data_dir = os.path.join(local_dir_path, 'aclImdb')\n",
    "# Remove the unused data folder.\n",
    "import shutil\n",
    "shutil.rmtree(os.path.join(data_dir, 'train/unsup'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "For this dataset, the data is already split into train and test.\n",
    "We just load them separately.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "print(data_dir)\n",
    "train_data = ak.text_dataset_from_directory(\n",
    "    os.path.join(data_dir, 'train'),\n",
    "    batch_size=batch_size)\n",
    "\n",
    "test_data = ak.text_dataset_from_directory(\n",
    "    os.path.join(data_dir, 'test'),\n",
    "    shuffle=False,\n",
    "    batch_size=batch_size)\n",
    "\n",
    "clf = ak.TextClassifier(overwrite=True, max_trials=1)\n",
    "clf.fit(train_data, epochs=2)\n",
    "print(clf.evaluate(test_data))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Load Data with Python Generators\n",
    "If you want to use generators, you can refer to the following code.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "N_BATCHES = 30\n",
    "BATCH_SIZE = 100\n",
    "N_FEATURES = 10\n",
    "\n",
    "\n",
    "def get_data_generator(n_batches, batch_size, n_features):\n",
    "    \"\"\"Get a generator returning n_batches random data of batch_size with n_features.\"\"\"\n",
    "\n",
    "    def data_generator():\n",
    "        for _ in range(n_batches * batch_size):\n",
    "            x = np.random.randn(n_features)\n",
    "            y = x.sum(axis=0) / n_features > 0.5\n",
    "            yield x, y\n",
    "\n",
    "    return data_generator\n",
    "\n",
    "\n",
    "dataset = tf.data.Dataset.from_generator(\n",
    "    get_data_generator(N_BATCHES, BATCH_SIZE, N_FEATURES),\n",
    "    output_types=(tf.float32, tf.float32),\n",
    "    output_shapes=((N_FEATURES,), tuple()),\n",
    ").batch(BATCH_SIZE)\n",
    "\n",
    "clf = ak.StructuredDataClassifier(overwrite=True, max_trials=1, seed=5)\n",
    "clf.fit(x=dataset, validation_data=dataset, batch_size=BATCH_SIZE)\n",
    "print(clf.evaluate(dataset))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Reference\n",
    "[image_dataset_from_directory](utils/#image_dataset_from_directory-function)\n",
    "[text_dataset_from_directory](utils/#text_dataset_from_directory-function)\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "load",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}